Out of balance

Code

library(tidyverse)
library(ospowertrader)
library(lubridate)
library(timetk)
library(arrow)

Picasso participation of Terna

Exploring the link between imbalance prices in the APG control area and Terna

Code

apg_prices <- arrow::read_parquet("./data/apg_imbalance_prices.parquet")
terna_prices <- arrow::read_parquet("./data/terna_imbalance_prices.parquet")

combined_prices <- apg_prices %>% 
  inner_join(terna_prices, by = c("df" = "reference_date")) %>% 
  select(df, apg_price = ae_price, terna_price = unbalance_price_EURxMWh) 

combined_prices %>% 
  pivot_longer(-df) %>% 
  timetk::plot_time_series(
    df, value, name, .smooth = FALSE, .title = "APG vs Terna (1)"
  )

This graph shows pretty well the impact of Terna’s participation to the Picasso platform. While large spikes, which were already common in the Austrian Control Area, were unheard of in Italy, after the 19th of July they became relatively common in Italy too.

Code

combined_prices %>% 
  ggplot(aes(apg_price, terna_price))+
  stat_binhex(bins = 50)+
  scale_fill_gradient(low = "blue", high = "red")+
  theme_bw()+
  ggtitle("APG vs Terna (2)")

Here we can see there is a lot of covariance for extreme values, notice (!) the difference in scale. APG tends to present much extremer imbalance prices.

Code

combined_prices %>% 
  filter(abs(apg_price) < 300 & abs(terna_price) < 300) %>% 
  ggplot()+
  geom_histogram(aes(apg_price), alpha = 0.8, fill = "red", bins = 200)+
  geom_histogram(aes(terna_price), alpha = 0.2, colour = "lightblue", bins = 200)+
  theme_bw()+
  ggtitle(label = "APG vs Terna (distribution)")

Very surprising. I have no idea why the distribution is so different.

Under normal circumstances/normal price levels for imbalance prices - APG seems to have a much smoother distribution. Terna presents more of a multimodal distribution, peaking around zero and two more peaks - probably values concentrate around the MGP/day-ahead price.

Picasso data

From the transnetbw.de website we can download price and volume data for the PICASSO platform. The problem working with Picasso data is the very granular nature of the datasets, prices and volumes are published in 4 seconds intervals. In order to work with the data it is necessary to aggregate to quarthourly granularity (to see what approach is used for aggregation see the code section).

Code

prices <- arrow::open_dataset("./data/picasso/prices") %>% 
  select(dt, terna_pos, terna_neg, apg_pos, apg_neg, qo)

volumes <- arrow::open_dataset("./data/picasso/volumes") %>% 
  select(dt, terna, apg, qo)


data <- prices %>% 
  left_join(volumes, by = c("dt", "qo")) %>% 
  collect()


# i made a mistake while downloading 
# i should have parsed the single files to convert to numeric
# now i have to do it manually

## raw picasso: splice volumes in order to avoid netting of volumes (negative and positive)
picasso_df <- data %>% 
  mutate(
    terna_pos = as.numeric(terna_pos), 
    terna_neg = as.numeric(terna_neg), 
    terna = as.numeric(terna),
    apg = as.numeric(apg),
    apg_pos = as.numeric(apg_pos),
    apg_neg = as.numeric(apg_neg)
    ) %>% 
  mutate(
    terna_pos_vol = if_else(terna > 0, terna, NA),
    terna_neg_vol = if_else(terna < 0, terna, NA),
    apg_pos_vol = if_else(apg > 0, apg, NA),
    apg_neg_vol = if_else(apg < 0, apg, NA)
  )

## calculate volume weighted prices


picasso_df_aggr <- picasso_df %>% 
  group_by(qo) %>% 
  summarise(
    terna_vw_price_neg = sum(terna_pos_vol * terna_neg, na.rm = TRUE)/sum(terna_pos_vol, na.rm = TRUE),
    terna_vw_price_pos = sum(terna_neg_vol * terna_pos, na.rm = TRUE)/sum(terna_neg_vol, na.rm = TRUE),
    apg_vw_price_pos   = sum(apg_pos_vol * terna_neg, na.rm = TRUE)/sum(apg_pos_vol, na.rm = TRUE),
    apg_vw_price_neg   = sum(apg_neg_vol * terna_pos, na.rm = TRUE)/sum(apg_neg_vol, na.rm = TRUE),
    ## calculate total volume called for both zones
    terna_total_pos_called = sum(terna_pos, na.rm  = TRUE) / n(),
    terna_total_neg_called = sum(terna_neg, na.rm = TRUE) / n(),
    apg_total_pos_called   = sum(apg_pos, na.rm = TRUE) / n(),
    apg_total_neg_called   = sum(apg_neg, na.rm = TRUE) / n()
  ) %>% 
  ungroup() %>% 
  left_join(combined_prices, by = c("qo" = "df"))

Code

# picasso_df_aggr - complete dataset - picasso and imbalanceprices
# first let's analyse terna and establish whether picasso prices influence imbalance prices

# prepare terna data

terna <- picasso_df_aggr %>% select(qo, starts_with("terna"))

terna_imbalance_volues <- arrow::read_parquet(file = "./data/terna_imbalance_volumes.parquet") %>% 
  select(reference_date, imbalance = zonal_aggregate_unbalance_MWh)

terna_splits_long_short <- terna %>% 
  left_join(terna_imbalance_volues, by = c("qo" = "reference_date")) %>% 
  mutate(imbalance_bin = if_else(imbalance > 0, "long", "short")) %>% 
  split(.$imbalance_bin)

Terna

Prices

First I investigate the relationship between realized Picasso prices and imbalance prices for each imbalance sign. In the following plot we look at quarter hours with a positive control area imbalance. Indeed there is a strong link between Picasso prices and imbalance prices in the control area.

Code

# we can see indeed the relationship of volume weighted picasso prices
# and imbalance prices

terna_splits_long_short$long %>% 
  ggplot(aes(terna_price, terna_vw_price_neg))+
  geom_point(alpha = 0.3) +
  theme_bw() +
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (Postive imbalance)")

Even looking at the less extreme values we see a strong correlation between Picasso prices and imbalance prices.

Code

terna_splits_long_short$long %>% 
  filter(terna_price > -100) %>% 
  filter(terna_vw_price_neg > -100) %>% 
  ggplot(aes(terna_price, terna_vw_price_neg))+
  geom_point(alpha = 0.1) +
  geom_smooth(method = "lm")+
  theme_bw() +
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (zoomed in)")

The same strong link is visible filtering for hours with negative control area imbalance.

Code

# the same as above
terna_splits_long_short$short %>% 
  ggplot(aes(terna_price, terna_vw_price_pos))+
  geom_point(alpha = 0.3)+
  theme_bw()+
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (negative imbalance)")

Code

terna_splits_long_short$short %>% 
  filter(terna_price < 400 & terna_vw_price_pos < 500) %>% 
  ggplot(aes(terna_price, terna_vw_price_pos))+
  geom_point(alpha = 0.1)+
  geom_smooth(method = "lm")+
  theme_bw() +
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (zoomed in)")

The relationship between volume weighted prices and imbalance prices is more complex in short hours. While there exists a relationship for some quarter hours, in others the imbalance price is completely detached, a Picasso price of 0 leads to a non zero imbalance price in the control area (all the points concentrating on the 0 value of the x axis).

Volumes

We can observe that the price for secondary reserve on the Picasso platform is highly dependent on the volume requested.

Code

terna_splits_long_short$long %>% 
  ggplot(aes(terna_vw_price_neg, terna_total_neg_called))+
  geom_point(alpha = 0.4)+
  theme_bw()+
  ggtitle("Total negative sec. reserve called vs price for sec. reserve")

Let’s zoom in to see the concentration of points in the top right corner.

Code

terna_splits_long_short$long %>%
  filter(terna_total_neg_called >= -300 & terna_vw_price_neg >= -300) %>% 
  ggplot(aes(terna_vw_price_neg, terna_total_neg_called))+
  geom_point(alpha = 0.2)+
  theme_bw()+
  ggtitle("Total negative sec. reserve called vs price for sec. reserve")

Code

terna_splits_long_short$short %>% 
  ggplot(aes(terna_vw_price_pos, terna_total_pos_called))+
  geom_point(alpha = 0.4)+
  theme_bw()+
  ggtitle("Total positive sec. reserve called vs price for sec. reserve")

The same strong link between called volumes and prices is true for upwards reserve.

Code

terna_splits_long_short$short %>%
  filter(terna_vw_price_pos < 1000) %>% 
  filter(terna_total_pos_called < 1000) %>% 
  ggplot(aes(terna_vw_price_pos, terna_total_pos_called))+
  geom_point(alpha = 0.4)+
  theme_bw()+
  ggtitle("Total positive sec. reserve called vs price for sec. reserve")

Like in the case of negative aFRR called also in case of calling positive aFRR there is almost a linear relationship between volumes and prices, with the exception of quite some quarter hours, that have a price of 0 while Terna is calling volumes from the Picasso platform.

Link to Imbalance prices in the control area NORD Terna

Code

terna_splits_long_short$long %>% 
  select(terna_total_neg_called, terna_price, imbalance, terna_vw_price_neg) %>% 
  GGally::ggpairs(alpha = 0.3)+
  theme_bw()

Code

terna_splits_long_short$short %>% 
  select(terna_total_pos_called, terna_price, imbalance, terna_vw_price_pos) %>% 
  GGally::ggpairs(alpha = 0.3)+
  theme_bw()

Both graphs show the strong link between imbalance prices and volume weighted picasso prices. However, we should keep in mind that it would be challenging working with this predictor, since picasso is not called upon in all quarter hours.

Code

terna_splits_long_short$long %>% 
  mutate(
    price_published = if_else(is.na(terna_vw_price_neg), 0, 1),
    volume_published = if_else(is.na(terna_total_neg_called), 0, 1)
  ) %>% 
  summarise(
    prices_published = sum(price_published)/n()
  ) %>% 
  mutate(across(everything(),~round(., 2)))

# A tibble: 1 × 1
  prices_published
             <dbl>
1             0.36

Price data is only available for roughly a third of quarter hours - at least in the period subject to this analyiss.

APG

Prices

Code

# prepare terna data

apg <- picasso_df_aggr %>% select(qo, starts_with("apg"))

apg_imbalance_volumes <- arrow::read_parquet(file = "./data/apg_control_area_imbalance.parquet") %>% 
  select(df, imbalance = cai)

apg_splits_long_short <- apg %>% 
  left_join(apg_imbalance_volumes, by = c("qo" = "df")) %>% 
  mutate(imbalance_bin = if_else(imbalance > 0, "long", "short")) %>% 
  split(.$imbalance_bin)

Code

apg_splits_long_short$long %>% 
  ggplot(aes(apg_price, apg_vw_price_neg))+
  geom_point(alpha = 0.3) +
  theme_bw() +
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (Postive imbalance)")

Code

apg_splits_long_short$long %>% 
  filter(apg_price > -400 & apg_vw_price_neg < 700) %>% 
  ggplot(aes(apg_price, apg_vw_price_neg))+
  geom_point(alpha = 0.3) +
  theme_bw() +
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (Postive imbalance)")

Code

apg_splits_long_short$short %>% 
  ggplot(aes(apg_price, apg_vw_price_pos))+
  geom_point(alpha = 0.3)+
  theme_bw()+
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (negative imbalance)")

Code

apg_splits_long_short$short %>% 
  filter(apg_price < 400 & apg_price > -100) %>% 
  filter(apg_vw_price_pos < 400 & apg_vw_price_pos > 0) %>% 
  ggplot(aes(apg_price, apg_vw_price_pos))+
  geom_point(alpha = 0.3)+
  theme_bw()+
  ggtitle("Volume weighted Picasso prices vs Imbalance Prices (negative imbalance)")

Overall the link between imbalance prices and picasso prices/volumes seems to be much much clearer in the case fo Terna. APG doesn’t present a strong and well identifiable link.

Volumes

Code

apg_splits_long_short$long %>% 
  ggplot(aes(apg_vw_price_neg, apg_total_neg_called))+
  geom_point(alpha = 0.4)+
  theme_bw()+
  ggtitle("Total negative sec. reserve called vs price for sec. reserve")

Exploring the volumetric link between APG and Terna

Here we can compare volumes - we detect the accession of Terna to the platfrom and also a pause in their participation.

Code

picasso_vol_data<- picasso_df %>% select(dt, contains("vol")) %>% 
  mutate(across(everything(), ~replace_na(., 0))) %>% 
  transmute(dt = dt, terna_vol = terna_pos_vol + terna_neg_vol, apg_vol = apg_pos_vol + apg_neg_vol) %>% 
  pivot_longer(-dt) %>% 
  drop_na() %>% 
  group_by(name) %>% 
  summarise_by_time(
    .date_var = dt, .by = "15 min", value = mean(value)
  ) %>% 
  ungroup() 

picasso_vol_data %>% 
  timetk::plot_time_series(dt, value, name, .smooth = FALSE, .title = "Volumes called by each TSO")

There seems to be a somewhat negative relationship between volumes. The two TSOs mirroring their volume calls on PICASSO.

Code

picasso_vol_data %>% 
  pivot_wider(names_from = name, values_from = value) %>% 
  filter(dt >= "2023-10-25") %>% 
  ggplot(aes(terna_vol, apg_vol))+
  geom_point(alpha = 0.3)+
  theme_bw()+
  ggtitle(label = "Volumes since end of October")

Example 26th of August 2023

An extreme day in terms of imbalance prices in Austria. We see too that volumes do somewhat correlate on this day. In particular there are 3 phases where Terna and APG call upon a similar level of volumes. For some periods there is a clear relationship between volumes and the imbalance price in Austria, however the very high imbalance prices from 8:00 to 8:30 fall out of this pattern.

Code

plot_volumes  <- picasso_df %>% 
  filter(date(qo) == "2023-08-26") %>% 
  select(dt, contains("vol")) %>% 
  pivot_longer(-dt) %>% 
  timetk::plot_time_series(
    dt, value, name, .smooth = FALSE,
    .title = "4s granularity - volumes data + imbalance prices /10",
    .interactive = FALSE
  )

imb_price <- picasso_df_aggr %>% 
  select(qo, apg_price) %>% 
  filter(date(qo) == "2023-08-26") %>% 
  mutate(apg_price = apg_price/10)

plot_vol_price <- plot_volumes + 
  geom_point(imb_price, mapping = aes(qo, apg_price))

plotly::ggplotly(plot_vol_price)